FRLG add selection arrow detection and start menu navigation

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_SelectionArrowDetector.cpp

@@ -0,0 +1,145 @@
+/*  Selection Arrow Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "Common/Cpp/Exceptions.h"
+#include "CommonTools/ImageMatch/WaterfillTemplateMatcher.h"
+#include "CommonTools/Images/WaterfillUtilities.h"
+#include "PokemonFRLG/PokemonFRLG_Settings.h"
+#include "PokemonFRLG_SelectionArrowDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+ImageFloatBox SelectionArrowDetector::arrow_box_for_position(SelectionArrowPositionStartMenu position){
+    switch (position){
+    case SelectionArrowPositionStartMenu::POKEDEX:
+        return ImageFloatBox(0.727692, 0.0523077, 0.0369231, 0.0778846);
+    case SelectionArrowPositionStartMenu::POKEMON:
+        return ImageFloatBox(0.727692, 0.1457692, 0.0369231, 0.0778846);
+    case SelectionArrowPositionStartMenu::BAG:
+        return ImageFloatBox(0.727692, 0.2392307, 0.0369231, 0.0778846);
+    case SelectionArrowPositionStartMenu::TRAINER:
+        return ImageFloatBox(0.727692, 0.3378846, 0.0369231, 0.0778846);
+    case SelectionArrowPositionStartMenu::SAVE:
+        return ImageFloatBox(0.727692, 0.4261538, 0.0369231, 0.0778846);
+    case SelectionArrowPositionStartMenu::OPTION:
+        return ImageFloatBox(0.727692, 0.5248076, 0.0369231, 0.0778846);
+    case SelectionArrowPositionStartMenu::EXIT:
+        return ImageFloatBox(0.727692, 0.6182692, 0.0369231, 0.0778846);
+    default:
+        break;
+    }
+    throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Invalid FRLG Selection Arrow Position");
+}
+
+ImageFloatBox SelectionArrowDetector::arrow_box_for_position(SelectionArrowPositionConfirmationMenu position) {
+    switch (position) {
+    case SelectionArrowPositionConfirmationMenu::YES:
+        return ImageFloatBox(0.69692, 0.4625, 0.037, 0.07788);
+    case SelectionArrowPositionConfirmationMenu::NO:
+        return ImageFloatBox(0.69692, 0.55, 0.037, 0.07788);
+    default:
+        break;
+    }
+    throw InternalProgramError(nullptr, PA_CURRENT_FUNCTION, "Invalid FRLG Selection Arrow Position");
+}
+
+class SelectionArrowMatcher : public ImageMatch::WaterfillTemplateMatcher{
+public:
+    SelectionArrowMatcher(const char* path)
+        : WaterfillTemplateMatcher(
+            path,
+            Color(50, 50, 50), Color(115, 115, 115), 70
+        )
+    {
+        m_aspect_ratio_lower = 0.8;
+        m_aspect_ratio_upper = 1.2;
+        m_area_ratio_lower = 0.8;
+        m_area_ratio_upper = 1.2;
+    }
+
+    static const SelectionArrowMatcher& matcher(){
+        static SelectionArrowMatcher matcher("PokemonFRLG/SelectionArrow.png");
+        return matcher;
+    }
+};
+
+SelectionArrowDetector::SelectionArrowDetector(
+    Color color,
+    VideoOverlay* overlay,
+    const ImageFloatBox& box
+)
+    : m_color(color)
+    , m_overlay(overlay)
+    , m_arrow_box(box)
+{}
+SelectionArrowDetector::SelectionArrowDetector(
+    Color color, 
+    VideoOverlay* overlay, 
+    SelectionArrowPositionStartMenu position
+)
+    : m_color(color)
+    , m_overlay(overlay)
+    , m_arrow_box(arrow_box_for_position(position))
+{}
+SelectionArrowDetector::SelectionArrowDetector(
+    Color color,
+    VideoOverlay* overlay,
+    SelectionArrowPositionConfirmationMenu position
+)
+    : m_color(color)
+    , m_overlay(overlay)
+    , m_arrow_box(arrow_box_for_position(position))
+{
+}
+void SelectionArrowDetector::make_overlays(VideoOverlaySet& items) const{
+    const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
+    items.add(m_color, GAME_BOX.inner_to_outer(m_arrow_box));
+}
+bool SelectionArrowDetector::detect(const ImageViewRGB32& screen){
+    const BoxOption& GAME_BOX = GameSettings::instance().GAME_BOX;
+    ImageViewRGB32 game_screen = extract_box_reference(screen, GAME_BOX);
+
+    double screen_rel_size = (game_screen.height() / 1080.0);
+    double screen_rel_size_2 = screen_rel_size * screen_rel_size;
+
+    double min_area_1080p = 700;
+    double rmsd_threshold = 80;
+    size_t min_area = size_t(screen_rel_size_2 * min_area_1080p);
+
+    const std::vector<std::pair<uint32_t, uint32_t>> FILTERS = {
+        {0xff464646, 0xff787878}
+    };
+
+    bool found = match_template_by_waterfill(
+        game_screen.size(),
+        extract_box_reference(game_screen, m_arrow_box),
+        SelectionArrowMatcher::matcher(),
+        FILTERS,
+        {min_area, SIZE_MAX},
+        rmsd_threshold,
+        [&](Kernels::Waterfill::WaterfillObject& object) -> bool {
+            m_last_detected = translate_to_parent(game_screen, m_arrow_box, object);
+            return true;
+        }
+    );
+
+    if (m_overlay){
+        if (found){
+            m_last_detected_box.emplace(*m_overlay, GAME_BOX.inner_to_outer(m_last_detected), COLOR_GREEN);
+        }else{
+            m_last_detected_box.reset();
+        }
+    }
+
+    return found;
+}
+
+
+}
+}
+}

SerialPrograms/Source/PokemonFRLG/Inference/PokemonFRLG_SelectionArrowDetector.h

@@ -0,0 +1,110 @@
+/*  Selection Arrow Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonFRLG_SelectionArrowDetector_H
+#define PokemonAutomation_PokemonFRLG_SelectionArrowDetector_H
+
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonTools/VisualDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonFRLG{
+
+const int START_MENU_OPTION_COUNT = 7;
+// The order of these enums should be the same as the order of options in the game menu, from top to bottom, for ease of use with loops.
+enum class SelectionArrowPositionStartMenu{
+    POKEDEX,
+    POKEMON,
+    BAG,
+    TRAINER,
+    SAVE,
+    OPTION,
+    EXIT
+};
+
+enum class SelectionArrowPositionConfirmationMenu {
+    YES,
+    NO
+};
+
+class SelectionArrowDetector : public StaticScreenDetector{
+public:
+    SelectionArrowDetector(
+        Color color,
+        VideoOverlay* overlay,
+        const ImageFloatBox& box
+    );
+
+    SelectionArrowDetector(
+        Color color,
+        VideoOverlay* overlay,
+        SelectionArrowPositionStartMenu position
+    );
+
+    SelectionArrowDetector(
+        Color color,
+        VideoOverlay* overlay,
+        SelectionArrowPositionConfirmationMenu position
+    );
+
+    static ImageFloatBox arrow_box_for_position(SelectionArrowPositionStartMenu position);
+
+    static ImageFloatBox arrow_box_for_position(SelectionArrowPositionConfirmationMenu position);
+
+    const ImageFloatBox& last_detected() const { return m_last_detected; }
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+
+    //  This is not const so that detectors can save/cache state.
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+private:
+    friend class SelectionArrowWatcher;
+
+    const Color m_color;
+    VideoOverlay* m_overlay;
+    const ImageFloatBox m_arrow_box;
+
+    ImageFloatBox m_last_detected;
+    std::optional<OverlayBoxScope> m_last_detected_box;
+};
+class SelectionArrowWatcher : public DetectorToFinder<SelectionArrowDetector>{
+public:
+    SelectionArrowWatcher(
+        Color color,
+        VideoOverlay* overlay,
+        const ImageFloatBox& box,
+        std::chrono::milliseconds hold_duration = std::chrono::milliseconds(250)
+    )
+         : DetectorToFinder("SelectionArrowWatcher", hold_duration, color, overlay, box)
+    {}
+    SelectionArrowWatcher(
+        Color color,
+        VideoOverlay* overlay,
+        SelectionArrowPositionStartMenu position,
+        std::chrono::milliseconds hold_duration = std::chrono::milliseconds(250)
+    )
+        : DetectorToFinder("SelectionArrowWatcher", hold_duration, color, overlay, arrow_box_for_position(position))
+    {
+    }
+    SelectionArrowWatcher(
+        Color color,
+        VideoOverlay* overlay,
+        SelectionArrowPositionConfirmationMenu position,
+        std::chrono::milliseconds hold_duration = std::chrono::milliseconds(250)
+    )
+        : DetectorToFinder("SelectionArrowWatcher", hold_duration, color, overlay, arrow_box_for_position(position))
+    {
+    }
+};
+
+
+}
+}
+}
+
+#endif

SerialPrograms/Source/PokemonFRLG/Programs/Farming/PokemonFRLG_NuggetBridgeFarmer.cpp

@@ -14,6 +14,8 @@
 #include "PokemonFRLG/Inference/Menus/PokemonFRLG_StartMenuDetector.h"
 #include "PokemonFRLG/PokemonFRLG_Navigation.h"
 #include "PokemonFRLG_NuggetBridgeFarmer.h"
+#include "PokemonFRLG/Inference/PokemonFRLG_SelectionArrowDetector.h"
+#include <PokemonFRLG/Programs/PokemonFRLG_StartMenuNavigation.h>
 
 namespace PokemonAutomation{
 namespace NintendoSwitch{
@@ -55,17 +57,17 @@ NuggetBridgeFarmer::NuggetBridgeFarmer()
         "<b>Number of Nuggets:</b><br>"
         "Zero will run until 'Stop after Current Nugget' is pressed or the program is manually stopped.",
         LockMode::UNLOCK_WHILE_RUNNING, 
-        120, // About 2 hours of farming. 
+        100, // About 2 hours of farming. 
         0
     )
     , GO_HOME_WHEN_DONE(false)
-    /*, PERIODIC_SAVE(
+    , PERIODIC_SAVE(
         "<b>Periodically Save:</b><br>"
         "Save the game every this many nuggets. This reduces the loss to game crashes. Set to zero to disable.",
         LockMode::UNLOCK_WHILE_RUNNING,
-        10,
+        50,
         0
-    )*/
+    )
     , NOTIFICATION_STATUS_UPDATE("Status Update", true, false, std::chrono::seconds(3600))
     , NOTIFICATIONS({
         &NOTIFICATION_STATUS_UPDATE,
@@ -77,7 +79,7 @@ NuggetBridgeFarmer::NuggetBridgeFarmer()
 
     PA_ADD_OPTION(NUM_NUGGETS);
     PA_ADD_OPTION(GO_HOME_WHEN_DONE);
-    //PA_ADD_OPTION(PERIODIC_SAVE);
+    PA_ADD_OPTION(PERIODIC_SAVE);
 
     PA_ADD_OPTION(NOTIFICATIONS);
 }
@@ -89,13 +91,19 @@ void NuggetBridgeFarmer::program(SingleSwitchProgramEnvironment& env, ProControl
     NuggetBridgeFarmer_Descriptor::Stats& stats = env.current_stats<NuggetBridgeFarmer_Descriptor::Stats>();
     DeferredStopButtonOption::ResetOnExit reset_on_exit(STOP_AFTER_CURRENT);
 
-    //for (uint32_t nuggets_since_last_save = 0;; nuggets_since_last_save++) {
-    while (true) {
+    for (uint32_t nuggets_since_last_save = 0;; nuggets_since_last_save++) {
         send_program_status_notification(env, NOTIFICATION_STATUS_UPDATE);
         if (NUM_NUGGETS != 0 && stats.nuggets >= NUM_NUGGETS) {
             break;
         }
 
+        if (PERIODIC_SAVE != 0 && nuggets_since_last_save >= PERIODIC_SAVE) {
+            env.console.log("Saving game...");
+
+            save_game_to_overworld(env.console, context);
+            nuggets_since_last_save = 0;
+        }
+
         env.console.log("Exiting Pokemon Center...");
         while (true){
             BlackScreenWatcher pokemon_ceter_exit(COLOR_RED);
@@ -210,41 +218,6 @@ void NuggetBridgeFarmer::program(SingleSwitchProgramEnvironment& env, ProControl
             }
         }
 
-        //TODO: Implement periodic saving. The Save Menu keeps the last cursor position. Need to implement arrow dectection on the correct option
-        /*if (PERIODIC_SAVE != 0 && nuggets_since_last_save >= PERIODIC_SAVE) {
-            StartMenuWatcher start_menu = StartMenuWatcher(COLOR_RED);
-
-            env.console.log("Saving game...");
-
-            while (true)
-            {
-                int ret = run_until<ProControllerContext>(
-                    env.console, context,
-                    [](ProControllerContext& context) {
-                        pbf_press_button(context, BUTTON_PLUS, 320ms, 640ms);
-                        pbf_wait(context, 100ms);
-                        context.wait_for_all_requests();
-                    },
-                    { start_menu }
-                );
-
-                if (ret == 0) {
-                    break;
-                }
-            }
-
-            pbf_press_dpad(context, DPAD_DOWN, 320ms, 320ms);
-            pbf_press_dpad(context, DPAD_DOWN, 320ms, 320ms);
-            pbf_press_dpad(context, DPAD_DOWN, 320ms, 320ms);
-            pbf_press_dpad(context, DPAD_DOWN, 320ms, 320ms);
-
-            pbf_press_button(context, BUTTON_A, 320ms, 320ms);
-            pbf_press_button(context, BUTTON_A, 320ms, 320ms);
-
-            pbf_mash_button(context, BUTTON_B, 2000ms);
-            nuggets_since_last_save = 0;
-        }*/
-
         stats.nuggets++;
         env.update_stats();
 

SerialPrograms/Source/PokemonFRLG/Programs/Farming/PokemonFRLG_NuggetBridgeFarmer.h

@@ -41,7 +41,7 @@ class NuggetBridgeFarmer : public SingleSwitchProgramInstance {
     SimpleIntegerOption<uint32_t> NUM_NUGGETS;
     GoHomeWhenDoneOption GO_HOME_WHEN_DONE;
 
-    //SimpleIntegerOption<uint32_t> PERIODIC_SAVE;
+    SimpleIntegerOption<uint32_t> PERIODIC_SAVE;
 
     EventNotificationOption NOTIFICATION_STATUS_UPDATE;
     EventNotificationsOption NOTIFICATIONS;