Tmr2023

.gitignore

@@ -1,6 +1,6 @@
 #Folders and files to ignore
 __ignore__/
-
+CATKIN_IGNORE
 ### ROS ###
 /catkin_home/devel/
 /catkin_home/devel_isolated/
@@ -29,8 +29,12 @@ __ignore__/
 /catkin_home/src/action_selectors/scripts/DeepSpeech/models/baidu_en8k/params.tar.gz
 /catkin_home/src/action_selectors/scripts/DeepSpeech/models/baidu_en8k/README.md
 /catkin_home/src/action_selectors/scripts/DeepSpeech/models/baidu_en8k/vocab.txt
-/catkin_home/src/action_selectors/scripts/DeepSpeech/models/lm/lm.binary
+/catkin_home/src/action_selectors/scripts/DeepSpeech/models/lm
+/catkin_home/src/action_selectors/scripts/DeepSpeech/models/baidu_en8k
+/catkin_home/src/action_selectors/scripts/DeepSpeech/models/librispeech
+
 /catkin_home/src/action_selectors/data/AzureSTTAPIKey.txt
+/catkin_home/src/main_engine/data/OpenAiKey.txt
 
 ### BASE CONTROL ###
 base_control/test
@@ -47,6 +51,7 @@ wrokspace.code-workspace
 /object_detection/models/dependencies/TensorFlow
 /object_detection/models/dependencies/coco
 /object_detection/images/train
+__ignore__/*
 catkin_home/src/object_detector/models
 catkin_home/src/object_detector/resources
 
@@ -63,5 +68,12 @@ out.log
 *.pyc
 .vscode/
 venv/
+*.log
+
+
+# tiago simulation #
+
+
+*tiago_public_ws/
 
-# End 
+#End

Makefile

@@ -5,15 +5,22 @@
 #: Builds a Docker image with the corresponding Dockerfile file
 noetic.build:
 	@docker build -t ros:home -f docker/noetic/Dockerfile .
+noetic.tiago.build:
+	@docker build -t ros:hometgo -f docker/noetic/Dockerfile.tiago .
+noetic.jetson.build:
+	@docker build -t ros:home -f docker/noetic/Dockerfile.jetson .
 noetic.speech.build:
 	@docker build -t ros:home -f docker/noetic/Dockerfile.speech .
+noetic.speech.prod.build:
+	@docker login ghcr.io
+	@docker build -t ros:homesp -f docker/noetic/Dockerfile.speech.prod .
 noetic.navigation.build:
 	@docker build -t ros:home -f docker/noetic/Dockerfile.navigation .
 noetic.objectDetection.build:
 	@docker build -t ros:home -f docker/noetic/Dockerfile.objectDetection .
 noetic.objectDetection.prod.build:
 	@docker login ghcr.io
-	@docker build -t ros:home -f docker/noetic/Dockerfile.objectDetection.prod .
+	@docker build -t ros:homeod -f docker/noetic/Dockerfile.objectDetection.prod .
 
 #: Create Generic Docker container
 noetic.create: 
@@ -38,32 +45,58 @@ noetic.speech.create.intel:
 	@./docker/run_scripts/runIntelGpu.bash IS_SPEECH
 noetic.speech.create.nvidia: 
 	@./docker/run_scripts/runNvidiaGpu.bash IS_SPEECH
+
+#: Create Speech Docker container
+noetic.speech.prod.create: 
+	@./docker/run_scripts/runspeech.bash IS_SPEECH
+noetic.speech.prod.create.intel: 
+	@./docker/run_scripts/runIntelGpu.bash IS_SPEECH
+noetic.speech.prod.create.nvidia: 
+	@./docker/run_scripts/runNvidiaGpu.bash IS_SPEECH
 
 #: Create ObjectDetection Docker container
 noetic.objectDetection.create: 
-	@./docker/run_scripts/run.bash IS_OBJECT_DETECTION
+	@./docker/run_scripts/runobj.bash IS_OBJECT_DETECTION
 noetic.objectDetection.create.intel: 
 	@./docker/run_scripts/runIntelGpu.bash IS_OBJECT_DETECTION
 noetic.objectDetection.create.nvidia: 
 	@./docker/run_scripts/runNvidiaGpu.bash IS_OBJECT_DETECTION
 
 #: Create ObjectDetection Prod Docker container
 noetic.objectDetection.prod.create: 
-	@./docker/run_scripts/run.bash IS_OBJECT_DETECTION_PROD
+	@./docker/run_scripts/runobjdtc.bash IS_OBJECT_DETECTION_PROD
 noetic.objectDetection.prod.create.intel: 
 	@./docker/run_scripts/runIntelGpu.bash IS_OBJECT_DETECTION_PROD
 noetic.objectDetection.prod.create.nvidia: 
-	@./docker/run_scripts/runNvidiaGpu.bash IS_OBJECT_DETECTION_PROD
+	@./docker/run_scripts/runNvidiaGpuOd.bash IS_OBJECT_DETECTION_PROD
 
 #: Start the container in background
 noetic.up:
 	@xhost +
 	@docker start ros-home
 
+noetic.speech.up:
+	@xhost +
+	@docker start ros-homesp
+
+noetic.objectDetection.up:
+	@xhost +
+	@docker start ros-homeod
+
+noetic.tiago.up:
+	@xhost +
+	@docker start ros-hometgo
+
 #: Stop the container
 noetic.down:
 	@docker stop ros-home
 
+noetic.speech.down:
+	@docker stop ros-homesp
+
+noetic.objectDetection.down:
+	@docker stop ros-homeod
+
 #: Restarts the container
 noetic.restart:
 	@docker restart ros-home
@@ -76,10 +109,21 @@ noetic.logs:
 noetic.shell:
 	@docker exec -it ros-home bash
 
+noetic.speech.shell:
+	@docker exec -it ros-home bash
+
+noetic.tiago.shell:
+	@docker exec -it ros-hometgo bash
 #: Remove ros-home container. 
 noetic.remove: noetic.down
 	@docker container rm ros-home
 
+noetic.speech.remove: noetic.speech.down
+	@docker container rm ros-homesp
+
+noetic.objectDetection.remove: noetic.objectDetection.down
+	@docker container rm ros-homeod
+
 # ----------------------------------------------------------------------
 #  General Docker
 # ----------------------------------------------------------------------

catkin_home/run/setup_devices2022.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+## **This are the IPs & Hostnames used for setup during TMR2021.
+source ./devel/setup.bash
+ROS_IP=192.168.31.187
+export ROS_IP
+ROS_HOSTNAME="rbrgs"
+export ROS_HOSTNAME
+ROS_MASTER_URI=http://192.168.1.10:11311
+export ROS_MASTER_URI

catkin_home/src/.rosinstall

@@ -1,18 +1,103 @@
-# File used by the ROS command `wstool` to download packages
-# that the project needs.
-# http://wiki.ros.org/wstool
-# 
-# `wstool update -t src` automatically downloads packages listed 
-# here. They will be downloaded inside a ExternalPackages/ directory;
-# then, when running `catkin build`/`catkin_make`, those packages
-# will be also build, included in the same workspace, and
-# will be available like other packages declared there.
-#
-# To include a new one from the catkin_home directory run:
-# `wstool set ExternalPackages/*package's-drectory-name* --git *https://link.git* --version=*vrsn* -t src`
-#
-# NOTE: After running a `wstool set`, please add again this comment.
+Installation of tiago simulation packages
 
-- git:
-    local-name: ExternalPackages/vizbox
-    uri: https://github.com/RoboCupAtHome/vizbox.git
+# IT IS UNLIKELY YOU WANT TO EDIT THIS FILE BY HAND,
+# UNLESS FOR REMOVING ENTRIES.
+# IF YOU WANT TO CHANGE THE ROS ENVIRONMENT VARIABLES
+# USE THE rosinstall TOOL INSTEAD.
+# IF YOU CHANGE IT, USE rosinstall FOR THE CHANGES TO TAKE EFFECT
+- git: {local-name: ExternalPackages/vizbox, uri: 'https://github.com/RoboCupAtHome/vizbox.git'}
+- setup-file: {local-name: /opt/ros/noetic/setup.sh}
+- git: {local-name: aruco_ros, uri: 'https://github.com/pal-robotics/aruco_ros.git',
+    version: noetic-devel}
+- git: {local-name: backward_ros, uri: 'https://github.com/pal-robotics/backward_ros',
+    version: kinetic-devel}
+- git: {local-name: custom_end_effector, uri: 'https://github.com/pal-robotics/custom_end_effector.git',
+    version: master}
+- git: {local-name: ddynamic_reconfigure_python, uri: 'https://github.com/pal-robotics/ddynamic_reconfigure_python',
+    version: master}
+- git: {local-name: dynamic_introspection, uri: 'https://github.com/pal-robotics/dynamic_introspection',
+    version: kinetic-devel}
+- git: {local-name: eband_local_planner, uri: 'https://github.com/pal-robotics-forks/eband_local_planner.git',
+    version: melodic-devel}
+- git: {local-name: gazebo_ros_pkgs, uri: 'https://github.com/pal-robotics-forks/gazebo_ros_pkgs.git',
+    version: noetic-devel}
+- git: {local-name: head_action, uri: 'https://github.com/pal-robotics/head_action.git',
+    version: indigo-devel}
+- git: {local-name: hey5_description, uri: 'https://github.com/pal-robotics/hey5_description.git',
+    version: master}
+- git: {local-name: humanoid_msgs, uri: 'https://github.com/ahornung/humanoid_msgs.git',
+    version: master}
+- git: {local-name: navigation_layers, uri: 'https://github.com/DLu/navigation_layers.git',
+    version: noetic}
+- git: {local-name: omni_base_navigation, uri: 'https://github.com/pal-robotics/omni_base_navigation.git',
+    version: melodic-devel}
+- git: {local-name: omni_base_robot, uri: 'https://github.com/pal-robotics/omni_base_robot.git',
+    version: melodic-devel}
+- git: {local-name: omni_base_simulation, uri: 'https://github.com/pal-robotics/omni_base_simulation.git',
+    version: melodic-devel}
+- git: {local-name: openslam_gmapping, uri: 'https://github.com/ros-perception/openslam_gmapping.git',
+    version: melodic-devel}
+- git: {local-name: pal_gazebo_plugins, uri: 'https://github.com/pal-robotics/pal_gazebo_plugins.git',
+    version: melodic-devel}
+- git: {local-name: pal_gazebo_worlds, uri: 'https://github.com/pal-robotics/pal_gazebo_worlds.git',
+    version: kinetic-devel}
+- git: {local-name: pal_gripper, uri: 'https://github.com/pal-robotics/pal_gripper.git',
+    version: kinetic-devel}
+- git: {local-name: pal_hardware_gazebo, uri: 'https://github.com/pal-robotics/pal_hardware_gazebo.git',
+    version: melodic-devel}
+- git: {local-name: pal_hardware_interfaces, uri: 'https://github.com/pal-robotics/pal_hardware_interfaces.git',
+    version: indigo-devel}
+- git: {local-name: pal_msgs, uri: 'https://github.com/pal-robotics/pal_msgs.git',
+    version: indigo-devel}
+- git: {local-name: pal_navigation_cfg_public, uri: 'https://github.com/pal-robotics/pal_navigation_cfg_public.git',
+    version: master}
+- git: {local-name: pal_navigation_sm, uri: 'https://github.com/pal-robotics/pal_navigation_sm.git',
+    version: master}
+- git: {local-name: pal_python, uri: 'https://github.com/pal-robotics/pal_python.git',
+    version: melodic-devel}
+- git: {local-name: pal_robotiq_gripper, uri: 'https://github.com/pal-robotics/pal_robotiq_gripper.git',
+    version: master}
+- git: {local-name: pal_statistics, uri: 'https://github.com/pal-robotics/pal_statistics',
+    version: kinetic-devel}
+- git: {local-name: pal_wsg_gripper, uri: 'https://github.com/pal-robotics/pal_wsg_gripper',
+    version: indigo-devel}
+- git: {local-name: play_motion, uri: 'https://github.com/pal-robotics/play_motion.git',
+    version: kinetic-devel}
+- git: {local-name: pmb2_navigation, uri: 'https://github.com/pal-robotics/pmb2_navigation',
+    version: kinetic-devel}
+- git: {local-name: pmb2_robot, uri: 'https://github.com/pal-robotics/pmb2_robot.git',
+    version: kinetic-devel}
+- git: {local-name: pmb2_simulation, uri: 'https://github.com/pal-robotics/pmb2_simulation.git',
+    version: kinetic-devel}
+- git: {local-name: roboticsgroup_gazebo_plugins, uri: 'https://github.com/pal-robotics-forks/roboticsgroup_gazebo_plugins.git',
+    version: kinetic-devel}
+- git: {local-name: robot_pose, uri: 'https://github.com/pal-robotics/robot_pose',
+    version: kinetic-devel}
+- git: {local-name: ros_controllers, uri: 'https://github.com/pal-robotics-forks/ros_controllers.git',
+    version: kinetic-devel}
+- git: {local-name: ros_control, uri: 'https://github.com/pal-robotics-forks/ros_control.git',
+    version: kinetic-devel}
+- git: {local-name: rviz_plugin_covariance, uri: 'https://github.com/pal-robotics/rviz_plugin_covariance.git',
+    version: kinetic-devel}
+- git: {local-name: simple_grasping_action, uri: 'https://github.com/pal-robotics/simple_grasping_action.git',
+    version: master}
+- git: {local-name: simple_models_robot, uri: 'https://github.com/pal-robotics/simple_models_robot.git',
+    version: kinetic-devel}
+- git: {local-name: slam_gmapping, uri: 'https://github.com/ros-perception/slam_gmapping.git',
+    version: melodic-devel}
+- git: {local-name: teleop_tools, uri: 'https://github.com/pal-robotics-forks/teleop_tools.git',
+    version: pal-overlay-kinetic-devel}
+- git: {local-name: tf_lookup, uri: 'https://github.com/pal-robotics/tf_lookup.git',
+    version: master}
+- git: {local-name: tiago_description_calibration, uri: 'https://github.com/pal-robotics/tiago_description_calibration.git',
+    version: indigo-devel}
+- git: {local-name: tiago_moveit_config, uri: 'https://github.com/pal-robotics/tiago_moveit_config.git',
+    version: kinetic-devel}
+- git: {local-name: tiago_navigation, uri: 'https://github.com/pal-robotics/tiago_navigation.git',
+    version: kinetic-devel}
+- git: {local-name: tiago_robot, uri: 'https://github.com/pal-robotics/tiago_robot.git',
+    version: kinetic-devel}
+- git: {local-name: tiago_simulation, uri: 'https://github.com/pal-robotics/tiago_simulation.git',
+    version: kinetic-devel}
+- git: {local-name: tiago_tutorials, uri: 'https://github.com/pal-robotics/tiago_tutorials.git',
+    version: noetic-devel}

catkin_home/src/action_selectors/launch/conversation_speech.launch

@@ -1,12 +1,12 @@
 <launch>
     <!-- Devices -->
-    <!-- <node name="AudioCapturer" pkg="devices" type="AudioCapturer.py" respawn="true" output="screen"/>
+    <node name="AudioCapturer" pkg="devices" type="AudioCapturer.py" respawn="true" output="screen"/>
     <node name="UsefulAudio" pkg="devices" type="UsefulAudio.py" respawn="true" output="screen"/>
-    <node name="ImageCapturer" pkg="devices" type="ImageCapture.py" respawn="true">
+    <!-- <node name="ImageCapturer" pkg="devices" type="ImageCapture.py" respawn="true">
         <param name="RATE" value="15" />
         <param name="CAMERAID" value="2" />
-    </node>
-    <node name="say" pkg="devices" type="say.py" respawn="true" output="screen" /> -->
+    </node> -->
+    <node name="say" pkg="devices" type="say.py" respawn="true" output="screen" />
     <!-- Speech To Text -->
     <!-- FORCE_ENGINE: 'online' - Azure, 'offline' - DeepSpeech, 'none' - Both Engines [it depends on internet] -->
     <arg name="FORCE_ENGINE" default="online"/>
@@ -17,11 +17,11 @@
     <node name="DeepSpeech" pkg="action_selectors" type="DeepSpeech.py" output="screen" if="$(eval arg('FORCE_ENGINE') != 'online')"/>
     <!-- Conversation -->
     <!-- START_TALK: 'True' - Start Conversation, 'False' - Wait until being Spoken -->
-    <arg name="START_TALK" default="True"/>
-    <node name="SomeoneToTalk" pkg="action_selectors" type="SomeoneToTalk.py" respawn="true">
+    <arg name="START_TALK" default="False"/>
+    <!-- <node name="SomeoneToTalk" pkg="action_selectors" type="SomeoneToTalk.py" respawn="true">
         <param name="CAMERAID" value="1" />
-    </node>
-    <node name="parser" pkg="action_selectors" type="parser.py" respawn="true" output="screen">
+    </node> -->
+    <!-- <node name="parser" pkg="action_selectors" type="parser.py" respawn="true" output="screen">
         <param name="START_TALK" value="$(arg START_TALK)" />
-    </node>
+    </node> -->
 </launch>

catkin_home/src/action_selectors/scripts/DeepSpeech.py

@@ -23,7 +23,7 @@
 def on_audio_callback(data):
     rospy.loginfo("DeepSpeech computing...")
     unicode_text = asr_server.bytes_speech_to_text(data.data)
-    text = unicode_text.encode('ascii', 'ignore')
+    text = unicode_text.encode('ascii', 'ignore').decode("utf-8")
 
     if len(text) == 0 or text.isspace():
        rospy.loginfo("Audio is empty")

catkin_home/src/action_selectors/scripts/DeepSpeech/data/librispeech/librispeech.py

@@ -11,6 +11,7 @@
 import distutils.util
 import os
 import sys
+sys.path.append('/catkin_home/src/action_selectors/scripts/DeepSpeech')
 import argparse
 import soundfile
 import json

catkin_home/src/action_selectors/scripts/DeepSpeech/models/lm/download_lm.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+URL='https://github.com/diegocardozo97/DeepSpeech/releases/download/v0.1/language_model-mozilla.zip'
+TARGET=language_model-mozilla.zip
+echo "Download LibriSpeech model ..."
+wget $URL
+
+if [ $? -ne 0 ]; then
+    echo "Fail to download LibriSpeech model!"
+    exit 1
+fi
+unzip $TARGET
+rm $TARGET
+
+exit 0

catkin_home/src/action_selectors/src/azureSpeechText.cpp

@@ -32,6 +32,7 @@ void onAudioCallback(const audio_common_msgs::AudioData::ConstPtr msg){
 
     // Creates an instance of a speech config with specified subscription key and service region.
     auto config = SpeechConfig::FromSubscription(SpeechUtils.getAPI(), SpeechUtils.getRegion());
+    ROS_INFO_STREAM("API: " << SpeechUtils.getAPI());
 
     // Creates a speech recognizer using file as audio input.
     auto callback = make_shared<AudioInputBytes>(msg->data);