View Issue Details
ID | Project | Category | View Status | Date Submitted | Last Update |
---|---|---|---|---|---|
0002373 | FSSCP | speech | public | 2011-01-07 20:20 | 2015-09-23 21:36 |
Reporter | MadProf | Assigned To | MageKing17 | ||
Priority | normal | Severity | minor | Reproducibility | N/A |
Status | resolved | Resolution | fixed | ||
Product Version | 3.6.13 | ||||
Target Version | 3.7.4 | Fixed in Version | 3.7.3 | ||
Summary | 0002373: Voice recognition improvement patch | ||||
Description | Patch to improve and expand the voice recognition - only tested personally on Windows 7 1) changes to inproc process rather than a shared - works best with windows built in recognition app turned off, no windows commands by accident. 2) removes the need to pause, now can just say "Alpha 1 destroy my target" etc. 3) adds ETS, Weapon config & targeting commands, e.g. "Max Engines", "Target next bomb" | ||||
Additional Information | patch effectively replaces all of code/sound/voicrec.cpp and adds checks in code/hud/hudsquadmsg.cpp as ship and wing order settings where been bypassed | ||||
Tags | No tags attached. | ||||
2011-01-07 20:20
|
voicerec.patch (19,559 bytes)
Index: code/hud/hudsquadmsg.cpp =================================================================== --- code/hud/hudsquadmsg.cpp (revision 6929) +++ code/hud/hudsquadmsg.cpp (working copy) @@ -1103,7 +1103,8 @@ Assert ( ainfo->shipnum != -1 ); ship_team = Ships[ainfo->shipnum].team; // team of the ship issuing the message - + // skip if player cannot order this ship + if (command & Ships[shipnum].orders_accepted) { switch ( command ) { // value of k matches the #defines for ship messages case ATTACK_TARGET_ITEM: if ( Objects[ainfo->target_objnum].type == OBJ_SHIP ) { @@ -1287,7 +1288,7 @@ break; } - + } // handle case of messaging one ship. Deal with messaging all fighters next. if ( ai_mode != AI_GOAL_NONE ) { Assert(ai_submode != -1234567); @@ -1390,6 +1391,8 @@ Assert ( Wings[wingnum].ship_index[0] != -1 ); wing_team = Ships[Wings[wingnum].ship_index[0]].team; + // skip if player cannot order this wing + if (command & Ships[Wings[wingnum].ship_index[0]].orders_accepted) { switch ( command ) { // value of k matches the #defines for ship messages case ATTACK_TARGET_ITEM: if ( Objects[ainfo->target_objnum].type == OBJ_SHIP ) { @@ -1505,7 +1508,7 @@ break; } - + } if ( ai_mode != AI_GOAL_NONE ) { Assert(ai_submode != -1234567); ai_add_wing_goal_player( AIG_TYPE_PLAYER_WING, ai_mode, ai_submode, target_shipname, wingnum ); Index: code/sound/phrases.xml =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/xml Index: code/sound/voicerec.cpp =================================================================== --- code/sound/voicerec.cpp (revision 6929) +++ code/sound/voicerec.cpp (working copy) @@ -17,6 +17,7 @@ #ifdef LAUNCHER #include "stdafx.h" + #endif //LAUNCHER @@ -24,6 +25,7 @@ #include <sphelper.h> // Contains definitions of SAPI functions #include <stdio.h> + #include "voicerec.h" #include "grammar.h" @@ -32,22 +34,25 @@ #include "io/keycontrol.h" #include "playerman/player.h" #include "ship/ship.h" +#include "cfile/cfile.h" CComPtr<ISpRecoGrammar> p_grammarObject; // Pointer to our grammar object CComPtr<ISpRecoContext> p_recogContext; // Pointer to our recognition context CComPtr<ISpRecognizer> p_recogEngine; // Pointer to our recognition engine instance +CComPtr<ISpAudio> cpAudio; // Pointer for Audio Input Device const bool DEBUG_ON = false; +void doVid_Action(int action); + bool VOICEREC_init(HWND hWnd, int event_id, int grammar_id, int command_resource) { HRESULT hr = S_OK; - CComPtr<ISpAudio> cpAudio; while ( 1 ) { // create a recognition engine - hr = p_recogEngine.CoCreateInstance(CLSID_SpSharedRecognizer); + hr = p_recogEngine.CoCreateInstance(CLSID_SpInprocRecognizer); if ( FAILED( hr ) ) { MessageBox(hWnd,"Failed to create a recognition engine\n","Error",MB_OK); @@ -55,8 +60,10 @@ break; } + // create the command recognition context hr = p_recogEngine->CreateRecoContext( &p_recogContext ); + if ( FAILED( hr ) ) { MessageBox(hWnd,"Failed to create the command recognition context\n","Error",MB_OK); @@ -82,24 +89,49 @@ break; } - // Load our grammar, which is the compiled form of simple.xml bound into this executable as a + if ( FAILED( hr ) ) + { + printf("Failed to set recognition state"); + break; + } + // Load our grammar, from data\phrases.xml or if that doesn't exist from the compiled in // user defined ("SRGRAMMAR") resource type. hr = p_recogContext->CreateGrammar(grammar_id, &p_grammarObject); if (FAILED(hr)) { - MessageBox(hWnd,"Failed to load grammar\n","Error",MB_OK); + MessageBox(hWnd,"Failed to create grammar\n","Error",MB_OK); break; } - - hr = p_grammarObject->LoadCmdFromResource(NULL, MAKEINTRESOURCEW(command_resource), - L"SRGRAMMAR", MAKELANGID(LANG_NEUTRAL, SUBLANG_NEUTRAL), - SPLO_DYNAMIC); + hr = p_grammarObject->LoadCmdFromFile(L"data\\phrases.xml", SPLO_STATIC); if ( FAILED( hr ) ) { - MessageBox(hWnd,"Failed to load resource\n","Error",MB_OK); - break; - } + hr = p_grammarObject->LoadCmdFromResource(NULL, MAKEINTRESOURCEW(command_resource), + //L"SRGRAMMAR", MAKELANGID(PRIMARYLANGID(m_langid), SUBLANGID(m_langid)), + L"SRGRAMMAR", MAKELANGID(LANG_NEUTRAL, SUBLANG_NEUTRAL), + SPLO_DYNAMIC); + if ( FAILED( hr ) ) + { + MessageBox(hWnd,"Failed to load resource SRGRAMMAR\n","Error",MB_OK); + break; + } + } + + // Set input for recognitation to default audio input + SpCreateDefaultObjectFromCategoryId(SPCAT_AUDIOIN, &cpAudio); + hr = p_recogEngine->SetInput(cpAudio, TRUE); + if (FAILED(hr)) + { + MessageBox(hWnd, "Failed to get default audio in\n","Error",MB_OK); + break; + } + hr = p_recogEngine->SetRecoState( SPRST_ACTIVE ); + if (FAILED(hr)) + { + MessageBox(hWnd, "Failed to set RecoState to active\n","Error",MB_OK); + break; + } + // Set rules to active, we are now listening for commands hr = p_grammarObject->SetRuleState(NULL, NULL, SPRS_ACTIVE ); if ( FAILED( hr ) ) @@ -122,6 +154,10 @@ void VOICEREC_deinit() { + if ( cpAudio ) + { + cpAudio.Release(); + } // Release grammar, if loaded if ( p_grammarObject ) { @@ -213,196 +249,296 @@ // Get the phrase elements, one of which is the rule id we specified in // the grammar. Switch on it to figure out which command was recognized. if (SUCCEEDED(pPhrase->GetPhrase(&pElements))) - { + { +#ifndef NDEBUG if(DEBUG_ON) { WCHAR *pwszText; + char szText[255]; + int i; + pPhrase->GetText(SP_GETWHOLEPHRASE, SP_GETWHOLEPHRASE, TRUE, &pwszText, NULL); - MessageBoxW(NULL,pwszText,NULL,MB_OK); - } - - switch ( pElements->Rule.ulId ) - { - case VID_ShipName: + + memset(szText, 0, 255); + for (i=0;i<254;i++) { - int wingType = pElements->pProperties->vValue.ulVal; - int shipNum = pElements->pProperties->pNextSibling->vValue.ulVal; - - char shipName[NAME_LENGTH]; - sprintf(shipName,"%s %d", wing_names[wingType], shipNum); - - Msg_instance = ship_name_lookup(shipName); - - // Can't issue commands to yourself or to nobody - if(Msg_instance < 0 || Msg_instance == Player_obj->instance) + if (*(pwszText + i) == 0) { break; } + szText[i] = (char)(*(pwszText + i)); + szText[i+1] = 0; + } - if(!(Player->flags & PLAYER_FLAGS_MSG_MODE)) - { - hud_squadmsg_toggle(); - } + mprintf(( "recognized speech : %s \n", szText )); + mprintf(( "speech Rule.ulId : %d \n", pElements->Rule.ulId )); + mprintf(( "confidence: %f \n", pElements->pProperties->SREngineConfidence)); + //MessageBoxW(NULL,pwszText,NULL,MB_OK); + } +#endif + if (pElements->pProperties->SREngineConfidence >= 0.90) { + int part1, part2, part3; + part1 = part2 = part3 = -1; - hud_squadmsg_do_mode(SM_MODE_SHIP_COMMAND); - break; - } - case VID_WingName: + switch ( pElements->Rule.ulId ) { - int wingType = pElements->pProperties->vValue.ulVal; + // wing name (alpha ...) has been spoken + case VID_WingName: + { + // this phrase might may be a combination of three parts (wingname)(wing/number)(action) + part1 = pElements->pProperties->vValue.ulVal; + if (pElements->pProperties->pNextSibling) + { + part2 = pElements->pProperties->pNextSibling->vValue.ulVal; + if (pElements->pProperties->pNextSibling->pNextSibling) + { + part3 = pElements->pProperties->pNextSibling->pNextSibling->vValue.ulVal; + } + } + if (part2 == -1) + break; // no ship number or wing - Msg_instance = wing_lookup(wing_names[wingType]); - if (Msg_instance < 0) - break; + if (part2 == 0) + { + // player has said "wing" phrase + Msg_instance = wing_lookup(wing_names[part1]); + if (Msg_instance < 0) + break; - if(!(Player->flags & PLAYER_FLAGS_MSG_MODE)) - { - hud_squadmsg_toggle(); - } + if(!(Player->flags & PLAYER_FLAGS_MSG_MODE)) + { + hud_squadmsg_toggle(); + } - hud_squadmsg_do_mode(SM_MODE_WING_COMMAND); - break; - } + hud_squadmsg_do_mode(SM_MODE_WING_COMMAND); + } + else + { + // player has spoken a ship number + char shipName[20]; + sprintf(shipName,"%s %d", wing_names[part1], part2); + + Msg_instance = ship_name_lookup(shipName); + if (Msg_instance < 0) + break; - case VID_Action: - { - int action = pElements->pProperties->vValue.ulVal; + if(!(Player->flags & PLAYER_FLAGS_MSG_MODE)) + { + hud_squadmsg_toggle(); + } - // If menu is up - if(Player->flags & PLAYER_FLAGS_MSG_MODE) - { - switch(action) - { - case VID_DestoryTarget: Msg_shortcut_command = ATTACK_TARGET_ITEM; break; - case VID_DisableTarget: Msg_shortcut_command = DISABLE_TARGET_ITEM; break; - case VID_DisarmTarget: Msg_shortcut_command = DISARM_TARGET_ITEM; break; - case VID_DestroySubsys: Msg_shortcut_command = DISABLE_SUBSYSTEM_ITEM; break; - case VID_ProtectTarget: Msg_shortcut_command = PROTECT_TARGET_ITEM; break; - case VID_IgnoreTarget: Msg_shortcut_command = IGNORE_TARGET_ITEM; break; - case VID_FormWing: Msg_shortcut_command = FORMATION_ITEM; break; - case VID_CoverMe: Msg_shortcut_command = COVER_ME_ITEM; break; - case VID_EngageEnemy: Msg_shortcut_command = ENGAGE_ENEMY_ITEM; break; - case VID_Depart: Msg_shortcut_command = DEPART_ITEM; break; - default: Msg_shortcut_command = -1; break; + hud_squadmsg_do_mode(SM_MODE_SHIP_COMMAND); } - if(Msg_instance == MESSAGE_ALL_FIGHTERS || Squad_msg_mode == SM_MODE_ALL_FIGHTERS ) + // see if an action has been spoken as well + if (part3 == -1) + break; + } + + // ship\wing action spoken + case VID_Action: + { + int action; + if (part3 == -1) { - // nprintf(("warning", "VOICER hud_squadmsg_send_to_all_fighters\n")); - hud_squadmsg_send_to_all_fighters(Msg_shortcut_command); + action = pElements->pProperties->vValue.ulVal; } - else if(Squad_msg_mode == SM_MODE_SHIP_COMMAND) + else { - // nprintf(("warning", "VOICER msg ship %d\n", Msg_instance)); - hud_squadmsg_send_ship_command( Msg_instance, Msg_shortcut_command, 1 ); + action = part3; } - else if(Squad_msg_mode == SM_MODE_WING_COMMAND) + + doVid_Action(action); + + break; + } + + // These commands run no matter what, and will even bring up the menu + case VID_TopMenu: + { + int action = pElements->pProperties->vValue.ulVal; + bool msgWindow = false; + if (Player->flags & PLAYER_FLAGS_MSG_MODE) { - // nprintf(("warning", "VOICER msg wing %d\n", Msg_instance)); - hud_squadmsg_send_wing_command( Msg_instance, Msg_shortcut_command, 1 ); + msgWindow = true; } - else if(Squad_msg_mode == SM_MODE_REINFORCEMENTS ) + + // If the command window is not up, or it is and its a cancel request toggle + if((msgWindow && action == VID_Cancel) || (!msgWindow && action != VID_Cancel)) { + hud_squadmsg_toggle(); } - - hud_squadmsg_toggle(); - } - else - { + switch(action) - { - case VID_DestoryTarget: button_function(ATTACK_MESSAGE); break; - case VID_DisableTarget: button_function(DISABLE_MESSAGE); break; - case VID_DisarmTarget: button_function(DISARM_MESSAGE); break; - case VID_DestroySubsys: button_function(ATTACK_SUBSYSTEM_MESSAGE); break; - case VID_ProtectTarget: button_function(PROTECT_MESSAGE); break; - case VID_IgnoreTarget: button_function(IGNORE_MESSAGE); break; - case VID_FormWing: button_function(FORM_MESSAGE); break; - case VID_CoverMe: button_function(COVER_MESSAGE); break; - case VID_EngageEnemy: button_function(ENGAGE_MESSAGE); break; - case VID_Depart: button_function(WARP_MESSAGE); break; - } - } - - break; - } + { + case VID_Ships: + hud_squadmsg_do_mode( SM_MODE_SHIP_SELECT ); + break; - // These commands run no matter what, and will even bring up the menu - case VID_TopMenu: - { - int action = pElements->pProperties->vValue.ulVal; - bool msgWindow = false; - if (Player->flags & PLAYER_FLAGS_MSG_MODE) - { - msgWindow = true; - } + case VID_Wings: + hud_squadmsg_do_mode( SM_MODE_WING_SELECT ); + break; - // If the command window is not up, or it is and its a cancel request toggle - if((msgWindow && action == VID_Cancel) || (!msgWindow && action != VID_Cancel)) - { - hud_squadmsg_toggle(); - } - - switch(action) - { - case VID_Ships: - hud_squadmsg_do_mode( SM_MODE_SHIP_SELECT ); - break; + case VID_AllFighters: + case VID_AllWings: + Msg_instance = MESSAGE_ALL_FIGHTERS; + Squad_msg_mode= SM_MODE_ALL_FIGHTERS ; + //hud_squadmsg_msg_all_fighters(); - case VID_Wings: - hud_squadmsg_do_mode( SM_MODE_WING_SELECT ); - break; + // can have the action to perform spoken directly afterwards + if (pElements->pProperties->pNextSibling) { + doVid_Action(pElements->pProperties->pNextSibling->vValue.ulVal); + } - case VID_AllFighters: - case VID_AllWings: - // Msg_instance = MESSAGE_ALL_FIGHTERS; - // Squad_msg_mode == SM_MODE_ALL_FIGHTERS - hud_squadmsg_msg_all_fighters(); + // if(Msg_shortcut_command == -1) + // { + // hud_squadmsg_do_mode( SM_MODE_ALL_FIGHTERS ); + // } + break; - // if(Msg_shortcut_command == -1) - // { - // hud_squadmsg_do_mode( SM_MODE_ALL_FIGHTERS ); - // } - break; + case VID_Reinforcements: + hud_squadmsg_do_mode( SM_MODE_REINFORCEMENTS ); + break; - case VID_Reinforcements: - hud_squadmsg_do_mode( SM_MODE_REINFORCEMENTS ); - break; + case VID_SupportShip: + hud_squadmsg_do_mode( SM_MODE_REPAIR_REARM ); + break; - case VID_SupportShip: - hud_squadmsg_do_mode( SM_MODE_REPAIR_REARM ); - break; + case VID_AbortSupport: + hud_squadmsg_do_mode( SM_MODE_REPAIR_REARM_ABORT ); + break; - case VID_AbortSupport: - hud_squadmsg_do_mode( SM_MODE_REPAIR_REARM_ABORT ); - break; + case VID_More: + break; + } - case VID_More: - break; + break; } - break; - } + // phrases for tranferring shield engery to different locations + case VID_shields: + { + int action = pElements->pProperties->vValue.ulVal; + //mprintf(("Shield Transfer %d \n", action)); + switch(action) + { + case 0: button_function( SHIELD_XFER_TOP ); break; + case 1: button_function( SHIELD_XFER_LEFT ); break; + case 2: button_function( SHIELD_XFER_RIGHT ); break; + case 3: button_function( SHIELD_XFER_BOTTOM ); break; + case 4: button_function( SHIELD_EQUALIZE ); break; - - - /* - - - hud_squadmsg_do_mode( SM_MODE_SHIP_COMMAND ); // and move to a new mode - hud_squadmsg_do_mode( SM_MODE_WING_COMMAND ); // and move to a new mode -hud_squadmsg_do_mode( SM_MODE_SHIP_COMMAND ); + } -*/ + break; + } - } + case VID_speed: + case VID_targeting: + case VID_other: + { + // basic cheat, phrase as a value equilvent to the defines in ControlConfig/ContolsConfig.h + // it just calls the button_function with this value + int action = pElements->pProperties->vValue.ulVal; + //mprintf(("Targeting/speed %d \n", action)); + + if (action > -1) + { + button_function( action ); + } + break; + } + + // nearly the same as the previous except it has some extra entries for + // maximising/minimising energy + case VID_power: + { + int action = pElements->pProperties->vValue.ulVal; + + if (action >= INCREASE_WEAPON && action <= ETS_EQUALIZE) + { + button_function( action ); + } + else + // this is for the max/min engines etc. + { + for (int i=1; i<7; i++) + { + button_function( action - 132 ); + } + } + break; + } + + } + } // Free the pElements memory which was allocated for us ::CoTaskMemFree(pElements); } } +void doVid_Action(int action) +{ + // If menu is up + if(Player->flags & PLAYER_FLAGS_MSG_MODE) + { + switch(action) + { + case VID_DestoryTarget: Msg_shortcut_command = ATTACK_TARGET_ITEM; break; + case VID_DisableTarget: Msg_shortcut_command = DISABLE_TARGET_ITEM; break; + case VID_DisarmTarget: Msg_shortcut_command = DISARM_TARGET_ITEM; break; + case VID_DestroySubsys: Msg_shortcut_command = DISABLE_SUBSYSTEM_ITEM; break; + case VID_ProtectTarget: Msg_shortcut_command = PROTECT_TARGET_ITEM; break; + case VID_IgnoreTarget: Msg_shortcut_command = IGNORE_TARGET_ITEM; break; + case VID_FormWing: Msg_shortcut_command = FORMATION_ITEM; break; + case VID_CoverMe: Msg_shortcut_command = COVER_ME_ITEM; break; + case VID_EngageEnemy: Msg_shortcut_command = ENGAGE_ENEMY_ITEM; break; + case VID_Depart: Msg_shortcut_command = DEPART_ITEM; break; + default: Msg_shortcut_command = -1; break; + + } + + if(Msg_instance == MESSAGE_ALL_FIGHTERS || Squad_msg_mode == SM_MODE_ALL_FIGHTERS ) + { + // nprintf(("warning", "VOICER hud_squadmsg_send_to_all_fighters\n")); + hud_squadmsg_send_to_all_fighters(Msg_shortcut_command); + } + else if(Squad_msg_mode == SM_MODE_SHIP_COMMAND) + { + // nprintf(("warning", "VOICER msg ship %d\n", Msg_instance)); + hud_squadmsg_send_ship_command( Msg_instance, Msg_shortcut_command, 1 ); + } + else if(Squad_msg_mode == SM_MODE_WING_COMMAND) + { + // nprintf(("warning", "VOICER msg wing %d\n", Msg_instance)); + hud_squadmsg_send_wing_command( Msg_instance, Msg_shortcut_command, 1 ); + } + else if(Squad_msg_mode == SM_MODE_REINFORCEMENTS ) + { + } + + hud_squadmsg_toggle(); + } + else + { + switch(action) + { + case VID_DestoryTarget: button_function(ATTACK_MESSAGE); break; + case VID_DisableTarget: button_function(DISABLE_MESSAGE); break; + case VID_DisarmTarget: button_function(DISARM_MESSAGE); break; + case VID_DestroySubsys: button_function(ATTACK_SUBSYSTEM_MESSAGE); break; + case VID_ProtectTarget: button_function(PROTECT_MESSAGE); break; + case VID_IgnoreTarget: button_function(IGNORE_MESSAGE); break; + case VID_FormWing: button_function(FORM_MESSAGE); break; + case VID_CoverMe: button_function(COVER_MESSAGE); break; + case VID_EngageEnemy: button_function(ENGAGE_MESSAGE); break; + case VID_Depart: button_function(WARP_MESSAGE); break; + } + } +} + #endif // VOICER #endif // _WIN32 |
2011-01-07 20:21
|
|
|
A bit late for me to finally notice this Mantis issue, but these upgrades (and more) have been in master since https://github.com/scp-fs2open/fs2open.github.com/commit/26553f13273a8d6f8f7f956390e0b5cc531a55ae |
|
I should've taken a closer look at this before I closed it; this actually contains a separate set of changes from the ones we rescued from the old unstable branch, and at least some of them should probably still be incorporated. |
|
What changes should be incorporated? This is targeted for 3.7.4 so this might be a good time to review those changes. |
|
Pull request: https://github.com/scp-fs2open/fs2open.github.com/pull/356 I didn't include the confidence threshold for reasons enumerated in the commit log/PR description. |
|
PR merged. |
Date Modified | Username | Field | Change |
---|---|---|---|
2011-01-07 20:20 | MadProf | New Issue | |
2011-01-07 20:20 | MadProf | File Added: voicerec.patch | |
2011-01-07 20:21 | MadProf | File Added: phrases.xml.zip | |
2011-05-29 07:28 | Goober5000 | Status | new => assigned |
2011-05-29 07:28 | Goober5000 | Assigned To | => Goober5000 |
2015-08-29 17:34 | MageKing17 | Note Added: 0016765 | |
2015-08-29 17:34 | MageKing17 | Assigned To | Goober5000 => MageKing17 |
2015-08-29 17:34 | MageKing17 | Status | assigned => resolved |
2015-08-29 17:34 | MageKing17 | Resolution | open => fixed |
2015-08-29 17:34 | MageKing17 | Fixed in Version | => 3.7.3 |
2015-08-29 17:34 | MageKing17 | Target Version | => 3.7.4 |
2015-08-31 18:57 | MageKing17 | Note Added: 0016770 | |
2015-08-31 18:57 | MageKing17 | Status | resolved => feedback |
2015-08-31 18:57 | MageKing17 | Resolution | fixed => reopened |
2015-09-18 11:54 | m_m | Note Added: 0016781 | |
2015-09-23 00:22 | MageKing17 | Note Added: 0016786 | |
2015-09-23 21:00 | MageKing17 | Status | feedback => code review |
2015-09-23 21:00 | MageKing17 | Resolution | reopened => open |
2015-09-23 21:00 | MageKing17 | Fixed in Version | 3.7.3 => |
2015-09-23 21:36 | m_m | Note Added: 0016789 | |
2015-09-23 21:36 | m_m | Status | code review => resolved |
2015-09-23 21:36 | m_m | Fixed in Version | => 3.7.3 |
2015-09-23 21:36 | m_m | Resolution | open => fixed |