From 72ae55b41b8a88efd2795183ae4c42c0ab8f99fc Mon Sep 17 00:00:00 2001 From: Grant Riddle <54089123+skAdOOdlEs7@users.noreply.github.com> Date: Mon, 23 May 2022 17:15:15 -0600 Subject: [PATCH 1/5] Create readme.md added directions for video tools --- misty_video_tools/readme.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 misty_video_tools/readme.md diff --git a/misty_video_tools/readme.md b/misty_video_tools/readme.md new file mode 100644 index 0000000..962a6e8 --- /dev/null +++ b/misty_video_tools/readme.md @@ -0,0 +1,7 @@ +# How to use video stream +- upload and run the video tools script to the misty +- open the VLC media player +- Go to media > stream +- click on the network tab +- where it asks for a url enter rtsp://*__ip of the robot__*:1935 +- click the stream button near the bottom From d524ce4946a80669267c43e3a364317752a0e49a Mon Sep 17 00:00:00 2001 From: cogrpar Date: Mon, 20 Jun 2022 17:50:54 -0600 Subject: [PATCH 2/5] started adding other skills --- misty_human_interaction/HumanInteraction.js | 78 ++++++++++++++++++--- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/misty_human_interaction/HumanInteraction.js b/misty_human_interaction/HumanInteraction.js index 539f38f..c820397 100644 --- a/misty_human_interaction/HumanInteraction.js +++ b/misty_human_interaction/HumanInteraction.js @@ -1,3 +1,5 @@ +/* ----------------- MISTY DIAGNOSTIC CAPTURE ----------------- */ + // head pitch and yaw callback functions for face tracking function _headYaw(data) { misty.Set("headYaw", data.AdditionalResults[1], false); @@ -6,6 +8,10 @@ function _headPitch(data) { misty.Set("headPitch", data.AdditionalResults[1], false); } + + + +/* ----------------- MISTY INTERACTION SKILLS ----------------- */ function FaceDetect (data){ // debugging function used to test if camera is able to recognize face // to monitor filter misty debug messages by 'face_detect' @@ -25,32 +31,83 @@ function ReadOnFaceDetect (data, callbackArgs) { misty.Speak(callbackArgs[0]); } +function TrackFace(data){ + // function to track a person's face once they have been identified and misty has said hello + + const faceDetected = data.PropertyTestResults[0].PropertyParent.Label; + const bearing = data.PropertyTestResults[0].PropertyParent.Bearing/2; // -13 right and +13 left + const elevation = data.PropertyTestResults[0].PropertyParent.Elevation/2; // -13 up and +13 down + misty.Debug(faceDetected + " detected"); + + const headYaw = misty.Get("headYaw"); + const headPitch = misty.Get("headPitch"); + const yawRight = misty.Get("yawRight"); + const yawLeft = misty.Get("yawLeft"); + const pitchUp = misty.Get("pitchUp"); + const pitchDown = misty.Get("pitchDown"); + + if (bearing != 0 && !(elevation < 7 && elevation > -7)) { // move misty's head so that it is oriented towards the user's face (this gets triggered if misty needs to reorient the pitch) + misty.MoveHeadDegrees(headPitch + ((pitchDown - pitchUp) / 66) * elevation, 0, headYaw + ((yawLeft - yawRight) / 132) * bearing, 100); // adjust pitch and yaw based on the location of the face (100% velocity) + } else if (bearing != 0) { + if (Math.abs(bearing) > 2){ // if the bearing is offset by more than from center, rotate the entire robot to face the person + var direction; + if (bearing > 0) { // positive bearing + direction = 1; + } + else { // negative bearing + direction = -1; + } + misty.DriveTime(0 /* linear velocity */, 100 * direction /* angular velocity */, 1500 /* time */); // rotate misty to the direction of the person + misty.MoveHeadDegrees(5 /* pitch */, 0 /* roll */, -((headYaw + (yawLeft - yawRight) / 132) * bearing)/40 /* yaw */, 100 /* velocity */); // rotate misty's head in the opposite direction to offset the rotation of the body + misty.Pause(2000); + } + else { + misty.MoveHeadDegrees(0, 0, headYaw + ((yawLeft - yawRight) / 132) * bearing, 100); + } + } else { + misty.MoveHeadDegrees(headPitch + ((pitchDown - pitchUp) / 66) * elevation, 0, 0, 100); + } + +} + + + + +/* ------------------- DEFAULT CONFIGURATION ------------------ */ +let basePitch = 0; +let skill = FaceDetect; +let callbackArgs = []; + + + + +/* ----------------------- LIBRARY CLASS ---------------------- */ class HumanInteraction { constructor() { // arguments of the form: this.basePitch=0, skill=this.FaceDetect, callbackArgs=[] - let basePitch = 0; - let skill = Skills.FaceDetect; - let callbackArgs = []; + /*let basePitch = 0; + let skill = FaceDetect; + let callbackArgs = [];*/ if (typeof arguments[0] == "number") { basePitch = arguments[0]; } else if (typeof arguments[0] == "function") { - this.skill = arguments[0]; + skill = arguments[0]; } else if (typeof arguments[0] == "object") { callbackArgs = arguments[0]; } if (typeof arguments[1] == "function") { - this.skill = arguments[1]; + skill = arguments[1]; } else if (typeof arguments[1] == "object") { callbackArgs = arguments[1]; } if (typeof arguments[2]) { - let callbackArgs = arguments[2]; + callbackArgs = arguments[2]; } // Global variable to store current pitch and yaw position of the head @@ -109,9 +166,14 @@ class HumanInteraction { } + + + + +/* ------------------ MAIN CALLBACK FUNCTION ------------------ */ function _FaceRec(data) { // FaceRec callback function misty.Debug("this part is working"); - FaceDetect(data); + skill(data, callbackArgs); } -const test = new HumanInteraction(0, HumanInteraction.FaceDetect); \ No newline at end of file +const test = new HumanInteraction(0, HumanInteraction.ReadOnFaceDetect, ["hello there, I saw your face!"]); From ecb146ff78f280aecf480b67272ef1eed079caab Mon Sep 17 00:00:00 2001 From: cogrpar Date: Mon, 18 Jul 2022 17:51:31 -0600 Subject: [PATCH 3/5] made some minor progress after weeks of immense pain and suffering --- misty_human_interaction/HumanInteraction.js | 83 ++++++++++++--------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/misty_human_interaction/HumanInteraction.js b/misty_human_interaction/HumanInteraction.js index c820397..03adccc 100644 --- a/misty_human_interaction/HumanInteraction.js +++ b/misty_human_interaction/HumanInteraction.js @@ -11,8 +11,17 @@ function _headPitch(data) { +//* ------------------- DEFAULT CONFIGURATION ------------------ */ +misty.Set("basePitch", [], false); +misty.Set("skill", "FaceDetect", false); +misty.Set("callbackArgs", [], false); +misty.Set("faceData", null); + + + + /* ----------------- MISTY INTERACTION SKILLS ----------------- */ -function FaceDetect (data){ +function _FaceDetect (data=misty.Get("data")){ // debugging function used to test if camera is able to recognize face // to monitor filter misty debug messages by 'face_detect' if (data.PropertyTestResults[0].PropertyParent.Label == "unknown person") { @@ -26,12 +35,14 @@ function FaceDetect (data){ misty.Debug("success"); } -function ReadOnFaceDetect (data, callbackArgs) { +function _ReadOnFaceDetect(data=misty.Get("data"), callbackArgs=misty.Get("callbackArgs")) { // function that causes misty to speak the passed callback args on a face being detected - misty.Speak(callbackArgs[0]); + misty.Speak(callbackArgs); // Speak is still not working on misty + misty.Debug("speaking: " + callbackArgs); + misty.Pause(1000); } -function TrackFace(data){ +function _TrackFace(data=misty.Get("data")){ // function to track a person's face once they have been identified and misty has said hello const faceDetected = data.PropertyTestResults[0].PropertyParent.Label; @@ -73,49 +84,37 @@ function TrackFace(data){ -/* ------------------- DEFAULT CONFIGURATION ------------------ */ -let basePitch = 0; -let skill = FaceDetect; -let callbackArgs = []; - - - - /* ----------------------- LIBRARY CLASS ---------------------- */ -class HumanInteraction { - constructor() { +function constructor() { // arguments of the form: this.basePitch=0, skill=this.FaceDetect, callbackArgs=[] - /*let basePitch = 0; - let skill = FaceDetect; - let callbackArgs = [];*/ if (typeof arguments[0] == "number") { - basePitch = arguments[0]; + misty.Set("basePitch", arguments[0], false); } - else if (typeof arguments[0] == "function") { - skill = arguments[0]; + else if (typeof arguments[0] == "string") { + misty.Set("skill", arguments[0], false); } else if (typeof arguments[0] == "object") { - callbackArgs = arguments[0]; + misty.Set("callbackArgs", arguments[0], false); } - if (typeof arguments[1] == "function") { - skill = arguments[1]; + if (typeof arguments[1] == "string") { + misty.Set("skill", arguments[1], false); } else if (typeof arguments[1] == "object") { - callbackArgs = arguments[1]; + misty.Set("callbackArgs", arguments[1], false); } if (typeof arguments[2]) { - callbackArgs = arguments[2]; + misty.Set("callbackArgs", arguments[2], false); } // Global variable to store current pitch and yaw position of the head // 'basePitch' is the default base angle of the pitch of the head (usually adjusted so to make misty look up if on the ground) misty.Debug("Centering Head"); - misty.MoveHeadDegrees(-basePitch, 0, 0, null, 0.5); + misty.MoveHeadDegrees(-misty.Get("basePitch"), 0, 0, null, 0.5); misty.Set("headYaw", 0.0, false); - misty.Set("headPitch", -basePitch, false); + misty.Set("headPitch", -misty.Get("basePitch"), false); // register listener for head yaw position from ActuatorPosition events function registerYaw() @@ -149,7 +148,7 @@ class HumanInteraction { } initiateHeadPhysicalLimitVariables(); - // define a function to register the face recognition events + /*// define a function to register the face recognition events function registerFaceRec(){ // Cancels any face recognition that's currently underway misty.StopFaceRecognition(); @@ -161,19 +160,33 @@ class HumanInteraction { misty.AddPropertyTest("FaceRec", "Label", "exists", "", "string"); // AddPropertyTest adds a test to determine which data will be sent to the event, in this case, if there is a person that goes with the detected face misty.RegisterEvent("FaceRec", "FaceRecognition", 1000, true); // RegisterEvent to register an event for face recognition (see callback function definition below) } - registerFaceRec(); - } - + registerFaceRec();*/ } +const test = constructor(0, "ReadOnFaceDetect", ["hello there, I saw your face!"]); +// define a function to register the face recognition events +function _registerFaceRec(){ + // Cancels any face recognition that's currently underway + misty.StopFaceRecognition(); + // Starts face recognition + misty.StartFaceRecognition(); + + misty.Debug("registered"); + + misty.AddPropertyTest("FaceRec", "Label", "exists", "", "string"); // AddPropertyTest adds a test to determine which data will be sent to the event, in this case, if there is a person that goes with the detected face + misty.RegisterEvent("FaceRec", "FaceRecognition", 1000, true); // RegisterEvent to register an event for face recognition (see callback function definition below) +} +//_registerFaceRec(); /* ------------------ MAIN CALLBACK FUNCTION ------------------ */ function _FaceRec(data) { // FaceRec callback function - misty.Debug("this part is working"); - skill(data, callbackArgs); + misty.Debug("this part is working........"); + misty.Debug("skill: " + misty.Get("skill")); + misty.RegisterTimerEvent(misty.Get("skill"), 800, false); + //misty.Pause(1000); + //misty.RegisterTimerEvent("registerFaceRec", 800, false); } - -const test = new HumanInteraction(0, HumanInteraction.ReadOnFaceDetect, ["hello there, I saw your face!"]); +_registerFaceRec(); From 28e1fca03a6209bcd7864d8ee560421cdcd40429 Mon Sep 17 00:00:00 2001 From: cogrpar Date: Mon, 1 Aug 2022 17:18:26 -0600 Subject: [PATCH 4/5] control flow logic is now successfully isolated from human interaction skill functions --- misty_human_interaction/HumanInteraction.js | 56 ++++++++++++--------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/misty_human_interaction/HumanInteraction.js b/misty_human_interaction/HumanInteraction.js index 03adccc..160b033 100644 --- a/misty_human_interaction/HumanInteraction.js +++ b/misty_human_interaction/HumanInteraction.js @@ -12,43 +12,50 @@ function _headPitch(data) { //* ------------------- DEFAULT CONFIGURATION ------------------ */ -misty.Set("basePitch", [], false); +misty.Set("basePitch", 0, false); misty.Set("skill", "FaceDetect", false); misty.Set("callbackArgs", [], false); -misty.Set("faceData", null); +misty.Set("faceDataLabel", "unknown person", false); +misty.Set("faceDataBearing", 0, false); // -13 right and +13 left +misty.Set("faceDataElevation", 0, false); // -13 up and +13 down /* ----------------- MISTY INTERACTION SKILLS ----------------- */ -function _FaceDetect (data=misty.Get("data")){ +function _FaceDetect (data){ // debugging function used to test if camera is able to recognize face // to monitor filter misty debug messages by 'face_detect' - if (data.PropertyTestResults[0].PropertyParent.Label == "unknown person") { + data = (typeof data.Label !== "undefined" && typeof data.bearing !== "undefined" && typeof data.elevation !== "undefined") ? data : {"Label": misty.Get("faceDataLabel"), "bearing": misty.Get("faceDataBearing"), "elevation": misty.Get("faceDataElevation")}; + + if (data.Label == "unknown person") { misty.Debug("face_detect: unknown face detected"); - misty.Debug(data); } else { misty.Debug("face_detect: known face detected"); - misty.Debug(data); } misty.Debug("success"); } -function _ReadOnFaceDetect(data=misty.Get("data"), callbackArgs=misty.Get("callbackArgs")) { +function _ReadOnFaceDetect(data, callbackArgs=misty.Get("callbackArgs")) { // function that causes misty to speak the passed callback args on a face being detected misty.Speak(callbackArgs); // Speak is still not working on misty misty.Debug("speaking: " + callbackArgs); - misty.Pause(1000); } -function _TrackFace(data=misty.Get("data")){ +function _TrackFace(data){ // function to track a person's face once they have been identified and misty has said hello - const faceDetected = data.PropertyTestResults[0].PropertyParent.Label; - const bearing = data.PropertyTestResults[0].PropertyParent.Bearing/2; // -13 right and +13 left - const elevation = data.PropertyTestResults[0].PropertyParent.Elevation/2; // -13 up and +13 down - misty.Debug(faceDetected + " detected"); + data = (typeof data.Label !== "undefined" && typeof data.bearing !== "undefined" && typeof data.elevation !== "undefined") ? data : {"Label": misty.Get("faceDataLabel"), "bearing": misty.Get("faceDataBearing"), "elevation": misty.Get("faceDataElevation")}; + + const faceDetect = data.Label; + const bearing = data.bearing; // -13 right and +13 left + const elevation = data.elevation; // -13 up and +13 down + + misty.Debug(bearing); + + const update = faceDetect + " detected, following face..."; + misty.Debug(update); const headYaw = misty.Get("headYaw"); const headPitch = misty.Get("headPitch"); @@ -78,6 +85,7 @@ function _TrackFace(data=misty.Get("data")){ } else { misty.MoveHeadDegrees(headPitch + ((pitchDown - pitchUp) / 66) * elevation, 0, 0, 100); } + misty.Pause(500); } @@ -163,10 +171,8 @@ function constructor() { registerFaceRec();*/ } -const test = constructor(0, "ReadOnFaceDetect", ["hello there, I saw your face!"]); - // define a function to register the face recognition events -function _registerFaceRec(){ +function registerFaceRec(debounce=250){ // Cancels any face recognition that's currently underway misty.StopFaceRecognition(); // Starts face recognition @@ -175,18 +181,22 @@ function _registerFaceRec(){ misty.Debug("registered"); misty.AddPropertyTest("FaceRec", "Label", "exists", "", "string"); // AddPropertyTest adds a test to determine which data will be sent to the event, in this case, if there is a person that goes with the detected face - misty.RegisterEvent("FaceRec", "FaceRecognition", 1000, true); // RegisterEvent to register an event for face recognition (see callback function definition below) + misty.RegisterEvent("FaceRec", "FaceRecognition", debounce, true); // RegisterEvent to register an event for face recognition (see callback function definition below) } -//_registerFaceRec(); /* ------------------ MAIN CALLBACK FUNCTION ------------------ */ function _FaceRec(data) { // FaceRec callback function - misty.Debug("this part is working........"); - misty.Debug("skill: " + misty.Get("skill")); + misty.Debug("running skill: " + misty.Get("skill")); + //misty.Set("faceData", data.PropertyTestResults[0].PropertyParent, false); + misty.Set("faceDataLabel", data.PropertyTestResults[0].PropertyParent.Label, false); + misty.Set("faceDataBearing", data.PropertyTestResults[0].PropertyParent.Bearing/2, false); // -13 right and +13 left + misty.Set("faceDataElevation", data.PropertyTestResults[0].PropertyParent.Elevation/2, false); // -13 up and +13 down + misty.RegisterTimerEvent(misty.Get("skill"), 800, false); - //misty.Pause(1000); - //misty.RegisterTimerEvent("registerFaceRec", 800, false); } -_registerFaceRec(); + +registerFaceRec(3000); +const test = constructor(30, "ReadOnFaceDetect", ["que pasa my home slice, chillin brutha?"]); + From fd532be70063c5892e9aae3d673d4015ec392d4c Mon Sep 17 00:00:00 2001 From: cogrpar Date: Mon, 1 Aug 2022 17:54:05 -0600 Subject: [PATCH 5/5] reframed as a class --- misty_human_interaction/HumanInteraction.js | 52 +++++++++------------ 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/misty_human_interaction/HumanInteraction.js b/misty_human_interaction/HumanInteraction.js index 160b033..fa26669 100644 --- a/misty_human_interaction/HumanInteraction.js +++ b/misty_human_interaction/HumanInteraction.js @@ -75,9 +75,9 @@ function _TrackFace(data){ else { // negative bearing direction = -1; } - misty.DriveTime(0 /* linear velocity */, 100 * direction /* angular velocity */, 1500 /* time */); // rotate misty to the direction of the person - misty.MoveHeadDegrees(5 /* pitch */, 0 /* roll */, -((headYaw + (yawLeft - yawRight) / 132) * bearing)/40 /* yaw */, 100 /* velocity */); // rotate misty's head in the opposite direction to offset the rotation of the body - misty.Pause(2000); + //misty.DriveTime(0 /* linear velocity */, 100 * direction /* angular velocity */, 1500 /* time */); // rotate misty to the direction of the person + misty.MoveHeadDegrees(5 /* pitch */, 0 /* roll */, -((headYaw + (yawLeft - yawRight) / 132) * bearing)/10 /* yaw */, 100 /* velocity */); // rotate misty's head in the opposite direction to offset the rotation of the body + //misty.Pause(2000); } else { misty.MoveHeadDegrees(0, 0, headYaw + ((yawLeft - yawRight) / 132) * bearing, 100); @@ -93,7 +93,8 @@ function _TrackFace(data){ /* ----------------------- LIBRARY CLASS ---------------------- */ -function constructor() { +class HumanInteraction { + constructor() { // arguments of the form: this.basePitch=0, skill=this.FaceDetect, callbackArgs=[] if (typeof arguments[0] == "number") { @@ -155,34 +156,23 @@ function constructor() { return 0; } initiateHeadPhysicalLimitVariables(); + } + + // define a method to register the face recognition events + registerFaceRec(debounce=250){ + // Cancels any face recognition that's currently underway + misty.StopFaceRecognition(); + // Starts face recognition + misty.StartFaceRecognition(); + + misty.Debug("registered"); + + misty.AddPropertyTest("FaceRec", "Label", "exists", "", "string"); // AddPropertyTest adds a test to determine which data will be sent to the ev + misty.RegisterEvent("FaceRec", "FaceRecognition", debounce, true); // RegisterEvent to register an event for face recognition (see callback func + } - /*// define a function to register the face recognition events - function registerFaceRec(){ - // Cancels any face recognition that's currently underway - misty.StopFaceRecognition(); - // Starts face recognition - misty.StartFaceRecognition(); - - misty.Debug("registered"); - - misty.AddPropertyTest("FaceRec", "Label", "exists", "", "string"); // AddPropertyTest adds a test to determine which data will be sent to the event, in this case, if there is a person that goes with the detected face - misty.RegisterEvent("FaceRec", "FaceRecognition", 1000, true); // RegisterEvent to register an event for face recognition (see callback function definition below) - } - registerFaceRec();*/ } -// define a function to register the face recognition events -function registerFaceRec(debounce=250){ - // Cancels any face recognition that's currently underway - misty.StopFaceRecognition(); - // Starts face recognition - misty.StartFaceRecognition(); - - misty.Debug("registered"); - - misty.AddPropertyTest("FaceRec", "Label", "exists", "", "string"); // AddPropertyTest adds a test to determine which data will be sent to the event, in this case, if there is a person that goes with the detected face - misty.RegisterEvent("FaceRec", "FaceRecognition", debounce, true); // RegisterEvent to register an event for face recognition (see callback function definition below) -} @@ -197,6 +187,6 @@ function _FaceRec(data) { // FaceRec callback function misty.RegisterTimerEvent(misty.Get("skill"), 800, false); } -registerFaceRec(3000); -const test = constructor(30, "ReadOnFaceDetect", ["que pasa my home slice, chillin brutha?"]); +const test = new HumanInteraction(10, "TrackFace", []); +test.registerFaceRec(800);